package pub.gavin.demo.crawler;

import com.jsoniter.JsonIterator;
import com.jsoniter.any.Any;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.apache.http.HttpEntity;
import org.apache.http.HttpHeaders;
import org.apache.http.HttpStatus;
import org.apache.http.NameValuePair;
import org.apache.http.client.entity.UrlEncodedFormEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.client.methods.HttpPost;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.message.BasicNameValuePair;
import org.apache.http.util.EntityUtils;

import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Map;

/**
 * Hello world!
 *
 */
public class App 
{
    public static void main( String[] args ) throws Exception
    {

        String dirName = DateFormatUtils.format(new Date(), DateFormatUtils.ISO_8601_EXTENDED_DATE_FORMAT.getPattern());
        FileUtils.forceMkdir(new File(dirName));

        //等待时间。单位：秒
        long sleepMillis = 0;

        CloseableHttpClient httpClient = HttpClients.createDefault();
        HttpPost httpPost = new HttpPost("https://www.lagou.com/jobs/positionAjax.json?px=default&city=广州&needAddtionalResult=false&isSchoolJob=0");

        //表单参数列表
        List<NameValuePair> nvps = new ArrayList<NameValuePair>();
        nvps.add(new BasicNameValuePair("first", "false"));
        nvps.add(new BasicNameValuePair("kd", "Java"));

        //爬取起始页页码
        int pn = 102;
        BasicNameValuePair basicNameValuePairPn = new BasicNameValuePair("pn", String.valueOf(pn));
        nvps.add(basicNameValuePairPn);

        //定义请求头
        httpPost.setEntity(new UrlEncodedFormEntity(nvps));
        httpPost.addHeader(HttpHeaders.ACCEPT,"application/json, text/javascript, */*; q=0.01");
        httpPost.addHeader(HttpHeaders.ACCEPT_ENCODING,"gzip, deflate, br");
        httpPost.addHeader(HttpHeaders.ACCEPT_LANGUAGE,"zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
        httpPost.addHeader(HttpHeaders.CONNECTION,"keep-alive");
        httpPost.addHeader(HttpHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=UTF-8");
        httpPost.addHeader(HttpHeaders.HOST, "www.lagou.com");
        httpPost.addHeader(HttpHeaders.REFERER, "https://www.lagou.com/jobs/list_Java");
        httpPost.addHeader(HttpHeaders.USER_AGENT,"Mozilla/5.0 (Windows NT 10.0; …) Gecko/20100101 Firefox/59.0");
        httpPost.addHeader("X-Anit-Forge-code","0");
        httpPost.addHeader("X-Anit-Forge-Token","None");
        httpPost.addHeader("X-Requested-With","XMLHttpRequest");

        //请求数据
        CloseableHttpResponse response = httpClient.execute(httpPost);
        //查询所有分页数据
        while (HttpStatus.SC_OK == response.getStatusLine().getStatusCode()) {

            HttpEntity entity = response.getEntity();
            String json = EntityUtils.toString(entity);
            EntityUtils.consume(entity);
            response.close();
            Map<String, Any> anyMap = JsonIterator.deserialize(json).asMap();

            Boolean success = anyMap.get("success").as(Boolean.class);
            if (success) {
                //显示请求结果
                System.out.print("等待" + sleepMillis + "秒，第" + pn + "页，请求结果：");
                System.out.println(json);
                //分析返回结果
                Map<String, Any> positionResult = anyMap.get("content").asMap().get("positionResult").asMap();
                List<Any> results = positionResult.get("result").asList();
                final Integer resultSize = positionResult.get("resultSize").as(Integer.class);
                if (resultSize.equals(0)) {
                    //如果当前页没有数据。说明已经爬取完毕了。
                    System.exit(0);
                }else {
                    //建立文档并记录信息
                    results.forEach(result -> {
                        String positionId = result.asMap().get("positionId").toString();
                        HttpGet httpGet = new HttpGet("https://www.lagou.com/jobs/" + positionId + ".html");
                        httpGet.addHeader(HttpHeaders.ACCEPT, "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
                        httpGet.addHeader(HttpHeaders.ACCEPT_ENCODING, "gzip, deflate, br");
                        httpGet.addHeader(HttpHeaders.ACCEPT_LANGUAGE, "zh-CN,zh;q=0.8,zh-TW;q=0.7,zh-HK;q=0.5,en-US;q=0.3,en;q=0.2");
                        httpGet.addHeader(HttpHeaders.CONNECTION, "keep-alive");
                        httpGet.addHeader(HttpHeaders.HOST, "www.lagou.com");
                        httpGet.addHeader(HttpHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0");
                        try ( CloseableHttpResponse positionResponse = httpClient.execute(httpGet) ) {
                            IOUtils.write(EntityUtils.toByteArray(positionResponse.getEntity()), FileUtils.openOutputStream(new File(new File(dirName), positionId + ".html")));
                            EntityUtils.consume(positionResponse.getEntity());
                        } catch (IOException e) {
                            e.printStackTrace();
                        }
                    });//companyId
                }
                //替换页码
                nvps.remove(basicNameValuePairPn);
                basicNameValuePairPn = new BasicNameValuePair("pn", String.valueOf(++pn));
                nvps.add(basicNameValuePairPn);
                httpPost.setEntity(new UrlEncodedFormEntity(nvps));
                //默认的每一页间的爬取间隔为5秒
                sleepMillis = 5;
            } else {
                //同一页，每次重爬，等待时间加1秒。
                sleepMillis++;
            }

            //加入等待时间
            Thread.sleep(sleepMillis * 1000);
            //请求数据
            response = httpClient.execute(httpPost);
        }
    }
}
